package Project;
import java.awt.RenderingHints.Key;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Array;
import java.sql.Time;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.LinkedList;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.Calendar;
import javax.swing.JApplet;
import javax.swing.JFileChooser;
import javax.swing.JTextArea;

public class preproc extends gui
{	
	// m - amount of malicious files 
	static int m = 0;
	// c - amount of benign files
	static int c = 0;
	public static final String DATE_FORMAT_NOW = "yyyy/MM/dd HH:mm:ss:ms";
	
	public static String now() 
	{
	    Calendar cal = Calendar.getInstance();
	    SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT_NOW);
	    return sdf.format(cal.getTime());
	 }
	
	public preproc(JTextArea log)
	{
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	Start Time");
			 outlog.newLine();
		
		
		String[] data = directoryFileIntoString(log);	
	
		// mfiles will hold the malicious files, its length is 2^10 + 1 = 1025
		String[] mfiles = new String[m];
		// cfiles will hold the clean and benign files, its length is 2^10 + 1 = 1025
		String[] cfiles = new String[c];
		int mf = 0;
		int cf = 0;
		for (int i=0; i<data.length; i++)
		{					
			if (data[i].endsWith("vir"))
			{
				mfiles[mf] = data[i];
				mf++;
			}
			else
			{
				cfiles[cf] = data[i];
				cf++;
			}
		}		
		// minimum occurrences
		int K = 5; 
				
		// hashGramCounterMal is a hash table of malicious sequence
		// hashGramCounterCln is a hash table of clean and benign sequence
		Hashtable hashGramCounterMal = new Hashtable();
		Hashtable hashGramCounterCln = new Hashtable();
		outlog.write(preproc.now()+"	calcWithHash for mal files");
		outlog.newLine();
		//System.out.println(preproc.now()+"	calcWithHash for mal files");
		// calculate the amount of malicious and benign grams in the files	
		calcWithHash(mfiles, hashGramCounterMal);
		outlog.write(preproc.now()+"	calcWithHash for cln files");
		outlog.newLine();
		//System.out.println(preproc.now()+"	calcWithHash for cln files");	
		calcWithHash(cfiles, hashGramCounterCln);
			
		int counter = 0;
		Object[] keys = hashGramCounterMal.keySet().toArray();
		outlog.write(preproc.now()+"	array of keys was created, going into entropy");
		outlog.newLine();
		//System.out.println(preproc.now()+"	array of keys was created, going into entropy");	
		try
		{
		    FileWriter fstreamEnt = new FileWriter("entropy.txt");
		    BufferedWriter outEnt = new BufferedWriter(fstreamEnt);
		    for (Object k : keys) 
		    {
				double entropy = 0;
				int malAmount = 0;
				if ((Integer)hashGramCounterMal.get(k)!=null)
					malAmount = (Integer)hashGramCounterMal.get(k);
				int clnAmount = 0;
				if ((Integer)hashGramCounterCln.get(k)!=null)
					clnAmount = (Integer)hashGramCounterCln.get(k);
				if (malAmount + clnAmount > K)
				{
					// calculating the entropy for each gram
				   	// entropy = [a/n]*[-p * log p] - [1-a/n]*[(1-p)*log(1-p)]
				   	// pplus = p ; pminus = 1-p
				   	double pplus = (double) malAmount / (malAmount + clnAmount);
				    double pminus = 1 - pplus;
				    // _pplus = a/n ; _pminus = 1-a/n
				    double _pplus = (double) (m - malAmount) / (c +
			        		m - malAmount - clnAmount);
				    double _pminus = 1 - _pplus;
				    // e1 = p*log p + (1-p)*log p
				    double e1 = pplus * java.lang.Math.log(pplus) 
		        			+ pminus * java.lang.Math.log(pminus);
				    double e2 = _pplus * java.lang.Math.log(_pplus) 
			    			+ _pminus * java.lang.Math.log(_pminus);
		     
					double p1 = (malAmount + clnAmount) / (c + m);
					double p2 = 1 - p1;
				    
					entropy = -(e1 * p1 + e2 * p2);
					long temp = (long) (entropy*java.lang.Math.pow(2,32));
				    	    
					outEnt.write((long)temp*(long)java.lang.Math.pow(2,24) + (Long)k + " "+(Long)k);
					outEnt.newLine();		       
				    counter++;
				}
			}		        
		    outEnt.close();
		    fstreamEnt.close();
		}
		catch (Exception e)
		{
			System.out.println("Error in entropy file");
		}
		outlog.write(preproc.now()+"	entropy was created");
		outlog.newLine();
		//System.out.println(preproc.now()+"	entropy was created");
		// grams is an array of top 1000 entropy values sorted from highest to lowest
		long [] grams = getEnt(counter);		
		long [] topKeys = getEntKeys(grams);
		outlog.write(preproc.now()+"	creating test and train files");
		outlog.newLine();
		//System.out.println(preproc.now()+"	creating test and train files");
		try
		{				
			// Create file Train
		    FileWriter fstream = new FileWriter("train.txt");
		    BufferedWriter out = new BufferedWriter(fstream);
		    out.write("@relation Train_70_Benign_AND_90_Malware");
		    out.newLine();
		        
		    // Create file Test
		    FileWriter fstreamTest = new FileWriter("test.txt");
		    BufferedWriter outTest = new BufferedWriter(fstreamTest);
		    outTest.write("@relation TEST_30_Benign_AND_10_Malware");
		    outTest.newLine();
		        
		    for (int top = 0; top<grams.length; top++)
		    {	        			   
		    	out.write("@attribute "+topKeys[top]+ " numeric");
		    	out.newLine();
			        
		    	outTest.write("@attribute "+topKeys[top]+ " numeric");
			    outTest.newLine();	
		    }	
	        	    
		    out.write("@attribute Class {B,M}");
			out.newLine();
			out.write("@data");
			out.newLine();
				
			outTest.write("@attribute Class {B,M}");
			outTest.newLine();
			outTest.write("@data");
			outTest.newLine();
				
			Double amountOfMal = 0.9*mf; 
			Double amountOfCln = 0.7*cf;
		
			int countMal = 0;
			int countCln = 0;
			outlog.write(preproc.now()+"	creating data part of files");
			outlog.newLine();
			//System.out.println(preproc.now()+"	creating data part of files");
			
			for (int index = 0; index < data.length; index++)
			{
				String line = new String();
				for (int gramIndex = 0; gramIndex < topKeys.length; gramIndex++)
				{
					try
					{			
						Vector <String> resVector = new Vector<String>() ;
						String str;
						FileReader fr = new FileReader(data[index]);
						BufferedReader in = new BufferedReader(fr);
						while ((str = in.readLine()) != null) 
					    {
							resVector.add(str); 
					    }
					    in.close();
						fr.close();
						str = resVector.toString();
						str = str.substring(1, str.length()-1);
										
						int v = getAmountOfGram(str, topKeys[gramIndex]);
						
						if ((line == null) && (line == ""))
							line = String.valueOf(v);
						else
							line = String.valueOf(v)+ "," + line;
					}
					catch (Exception e)
					{
						System.out.println("Exception in printing = "+e.getMessage());
					}
				}
				if ((data[index].endsWith("vir")) && (countMal < Math.round(amountOfMal)))
				{
					out.write(line + "M");
					out.newLine();					
					countMal++;
				}
				else
				{
					if (!(data[index].endsWith("vir")) && (countCln < Math.round(amountOfCln)))
					{
						out.write(line + "C");
						out.newLine();
						countCln++;						
					}					
				}				
				if ((data[index].endsWith("vir")) && (countMal > Math.round(amountOfMal)))
				{
					outTest.write(line + "M");
					outTest.newLine();					
					countMal++;
				}
				else
				{
					if (!(data[index].endsWith("vir")) && (countCln > Math.round(amountOfCln)))
					{
						outTest.write(line + "C");
						outTest.newLine();
						countCln++;						
					}					
				}
				if (countMal == Math.round(amountOfMal))
					countMal++;
				if (countCln == Math.round(amountOfCln))
					countCln++;
			}
			//Close the output stream
			out.close();
			outTest.close();			
		}
		catch (Exception e)
		{//Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
		outlog.write(preproc.now()+"	End Of Project");
		outlog.close();
		fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("error in log : "+ e.getMessage());
		}
		System.out.println(preproc.now()+"	End Of Project");
	} // end of preproc 
	

	// input : a vector of strings - files, and a vector of short numbers - gramCounter 
	// output : void 
	// this function calculates the amount of grams in the files
	private static void calcWithHash(String[] files, Hashtable hashGramCounter) 
	{		
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	enter calcWithHash");
			 outlog.newLine();
		
		//System.out.println(preproc.now()+"	enter calcWithHash");
		
		long k = 0;
		// for each file					
		for (int j = 0; j < files.length; j++) 
		{	// for each data line in the file	
			try
			{
				Vector <String> resVector = new Vector<String>() ;
				String str;
				FileReader fr = new FileReader(files[j]);
				BufferedReader in = new BufferedReader(fr);
				while ((str = in.readLine()) != null) 
			    {
					resVector.add(str);     
			    }
				str = resVector.toString();
				str = str.substring(1, str.length()-1);
				
			    in.close();
				fr.close();					
				for (int i = 0; i < str.length()-2; i++) 
				{				
					if (files[j].length() > 2)
					{
						char a = str.substring(i, i + 3).charAt(0);
						char b = str.substring(i, i + 3).charAt(1);
						char c = str.substring(i, i + 3).charAt(2);
						Boolean cont = true;
						if (str.substring(i, i + 3).contains(","))
							cont = false;					
						if (cont == true)
						{						
							k = Long.valueOf(a) * 256 + Long.valueOf(b) * 256 * 256 + Long.valueOf(c) ;							
							int v = 1;	
							if ((k != 0) && ((Integer)hashGramCounter.get(k) != null) && ((Integer)hashGramCounter.get(k) >= 1 ))							
								v = (Integer)hashGramCounter.get(k) + 1;
							hashGramCounter.put(k, v);					
						}
					}
				}
			}
			catch (Exception e)
			{
				System.out.println("Exception in calcWithHash = "+e.getMessage());
			}			
		}
		outlog.write(preproc.now()+"	exit calcWithHash");
		outlog.newLine();
		outlog.close();
		fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in calcWithHash = "+e.getMessage());
		}	
		//System.out.println(preproc.now()+"	exit calcWithHash");
	} // calcWithHash function
	
	public static int getAmountOfGram(String str, long key)
	{	
		/*
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	enter getAmountOfGram");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getAmountOfGram = "+e.getMessage());
		}
		*/
		int ans = 0;
		long k = 0;
		if (key != 0)
		{
			for (int i = 0; i < str.length() - 2; i++) 
			{
				char a = str.substring(i, i + 3).charAt(0);
				char b = str.substring(i, i + 3).charAt(1);
				char c = str.substring(i, i + 3).charAt(2);
				Boolean cont = true;
				if (str.substring(i, i + 3).contains(","))
					cont = false;				
				if (cont == true)
				{
					k = Long.valueOf(a) * 256 + Long.valueOf(b)	* 256 * 256 + Long.valueOf(c) ;
					if (k == key)				
						ans = ans + 1;
				}
			}
		}
		/*
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	exit getAmountOfGram");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getAmountOfGram = "+e.getMessage());
		}
		*/
		return ans;
	}
	
	// this file will open the files in the directory, place their data as string 
	// in an array of strings and returns that array

	// this file will open the files in the directory, place their data as string 
	// in an array of strings and returns that array
	/**
	 * will return all the files content. very memory intensive.
	 */
	public static String[] directoryFileIntoString(JTextArea log)
	{	
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	enter directoryFileIntoString");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in directoryFileIntoString = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	enter directoryFileIntoString");      
		
		String file = log.getText();		
		String[] f = file.split("\\n");
		for (int i = 0; i<f.length; i++)
		{
			if (f[i].endsWith("vir"))
				m++;
			else
				c++;
		}
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	exit directoryFileIntoString");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in directoryFileIntoString = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	exit directoryFileIntoString");
		return f;
	}
	
	public static long[] getEnt(int amount)
	{
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	enter getEnt");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getEnt = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	enter getEnt");
		long[] ent = new long[amount];
		long[] entAns = new long[1000];

		try
		{
			int i = 0;
			String str;
			FileReader fr = new FileReader("entropy.txt");
			BufferedReader in = new BufferedReader(fr);
			while ((str = in.readLine()) != null) 
		    {					
				StringTokenizer tok = new StringTokenizer(str);						
				ent[i] = Long.valueOf(tok.nextToken());		
				i++;
			}							
			in.close();
			fr.close();			
		}
		catch (Exception e)
		{
			System.out.println("Exception in getEnt = "+e.getMessage());
		}	
	
		//sort the entropy array (top values first)
		
		//Arrays.sort(ent);
		java.util.Arrays.sort(ent);
		int i = 0;
		while (i<1000)
		{
			//return only top 1000 entropies
			entAns[i] = ent[ent.length - 1 - i];
			i++;
		}
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	exit getEnt");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getEnt = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	exit getEnt");
		return entAns;
	}	

	public static long[] getEntKeys(long [] entropy)
	{
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	enter getEntKeys");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getEntKeys = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	enter getEntKeys");
		long[] key = new long[1000];
		
		for (int i = 0; i < 1000; i++)
		{
			try
			{
				String str;
				FileReader fr = new FileReader("entropy.txt");
				BufferedReader in = new BufferedReader(fr);
				while ((str = in.readLine()) != null) 
			    {
					StringTokenizer tok = new StringTokenizer(str);				
					String temp = tok.nextToken();
					if (entropy[i] == Long.valueOf(temp))  
					{
						key[i] = Long.valueOf(tok.nextToken());
						//System.out.println("e = "+entropy[i]+" k = "+key[i]);
					}
			    }							
			    in.close();
				fr.close();			
			}
			catch (Exception e)
			{
				System.out.println("Exception in getEntKeys = "+e.getMessage());
			}	
		}
		try
		{
			 FileWriter fstreamlog = new FileWriter("logFile.txt",true);
			 BufferedWriter outlog = new BufferedWriter(fstreamlog);
			 outlog.write(preproc.now()+"	exit getEntKeys");
			 outlog.newLine();
			 outlog.close();
			 fstreamlog.close();
		}
		catch (Exception e)
		{
			System.out.println("Exception in getEntKeys = "+e.getMessage());
		}
		//System.out.println(preproc.now()+"	exit getEntKeys");
		return key;
	}	
}


